{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm.auto import tqdm\n",
    "from collections import Counter\n",
    "from os import listdir\n",
    "import base64, json, csv\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_data(file_name, reset=False):\n",
    "    ret = pd.read_csv(file_name, sep='\\t')\n",
    "    return ret"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = '/tmp/r08922010/kdd2020/'\n",
    "valid = load_data(path+'valid.tsv')\n",
    "testA = load_data(path+'testA.tsv')\n",
    "testB = load_data(path+'testB.tsv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sig(x):\n",
    "    return 1/(1+np.e**(-x))\n",
    "\n",
    "def load_one_file(filename, na_val, pad_len, method):\n",
    "    preds = pd.read_csv(filename).fillna(na_val)\n",
    "    qid2score = {}\n",
    "    \n",
    "    for row in preds.values:\n",
    "        pid, score = row[1:pad_len+1], row[pad_len+1:]\n",
    "        l = len([x for x in pid if x != na_val])\n",
    "        if method == 'vote': qid2score[row[0]] = {pid[i]: weight[i] for i in range(l)}\n",
    "        else: qid2score[row[0]] = {pid[i]: score[i] for i in range(l)}\n",
    "        \n",
    "    return qid2score\n",
    "\n",
    "def ensemble(filenames, na_val, pad_len, thd, method='vote'):\n",
    "    counts = Counter(valid['product_id'].values.tolist()+testA['product_id'].values.tolist()+testB['product_id'].values.tolist())\n",
    "    print('loading files...')\n",
    "    # first file\n",
    "    qid2score = load_one_file(path+filenames[0], na_val, pad_len, method)\n",
    "    # other files\n",
    "    for filename in tqdm(filenames[1:]):\n",
    "        q2s = load_one_file(path+filename, na_val, pad_len, method)\n",
    "        # update score\n",
    "        for qid in list(qid2score.keys()):\n",
    "            for pid in list(qid2score[qid].keys()):\n",
    "                qid2score[qid][pid] += q2s[qid][pid]\n",
    "    # ensemble\n",
    "    print('ensembling...')\n",
    "    preds = {}\n",
    "    for qid in tqdm(list(qid2score.keys())):\n",
    "        pred = [(pid, qid2score[qid][pid]) for pid in list(qid2score[qid].keys())]\n",
    "        tmp_thd = thd\n",
    "        pred2 = [x for x in pred if counts[x[0]] <= tmp_thd]\n",
    "        while len(pred2) < 5:\n",
    "            tmp_thd += 1\n",
    "            pred2 = [x for x in pred if counts[x[0]] <= tmp_thd]\n",
    "        pred2.sort(key=lambda x: x[1], reverse=True)\n",
    "        preds[int(qid)] = [int(pid) for pid, _ in pred2[:5]]\n",
    "    return preds\n",
    "\n",
    "def nDCG_score(preds, answers):\n",
    "    iDCG = sum([sum([np.log(2)/np.log(i+2) for i in range(min(len(answer), 5))]) \\\n",
    "                for answer in list(answers.values())])\n",
    "    DCG = sum([sum([np.log(2)/np.log(i+2) if preds[qid][i] in answers[str(qid)] else 0 \\\n",
    "                    for i in range(len(preds[qid]))]) for qid in list(preds.keys())])\n",
    "    return DCG/iDCG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading files...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c9df3cc0b8cf4527a1da55347e04ab90",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=68), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "ensembling...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5aac279225964874b4446513c3833888",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=994), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "path = 'predictions/'\n",
    "mcans = {5119: 38, 519: 39, 123: 36, 1234: 36, 12345: 31, 1213: 34, 207: 34,\n",
    "         1333: 37, 2020: 36, 1115: 36, 666: 32, 2574: 39, 89983: 38, 46555: 38, 86031: 39,\n",
    "         7414: 35, 71438: 38, 777: 35, 87: 35, 8787: 32, 878787: 30, 800061: 31, 856710: 31,\n",
    "         42: 38, 426: 36, 64: 38, 8864: 36, 26: 39, 7382: 39, 1010: 39, 1001: 36,\n",
    "         2330: 37, 612: 39, 24: 38, 25: 32, 2077: 35, 2049: 39, 2045: 39, 1917: 36,\n",
    "         78667: 36, 68654: 34, 56474: 33, 56464: 36, 54367: 37, 4547: 32, 437: 36, 485: 38,\n",
    "         132: 38, 257: 37, 584: 35, 931: 37, 792: 33, 603: 39, 746: 39, 480: 35}\n",
    "visuals = {413: 32, 807: 37, 9527: 38, 713: 36, 625: 38, 1324: 38,\n",
    "           987: 34, 116: 39, 41: 30, 145: 39,\n",
    "           7328: 32, 62: 35, 3951: 37, 9736: 38}\n",
    "f_m = ['prediction_all_cls2_{}_{}.csv'.format(seed, mcans[seed]) \\\n",
    "       for seed in list(mcans.keys()) if mcans[seed]]\n",
    "f_v = ['prediction_all_cls2_{}_{}.csv'.format(seed, visuals[seed]) \\\n",
    "       for seed in list(visuals.keys()) if visuals[seed]]\n",
    "add_visual = True\n",
    "filenames = f_m+f_v if add_visual else f_m\n",
    "weight = [1/i for i in range(1, 31)]\n",
    "na_val = -1e10\n",
    "pad_len = 30\n",
    "thd = 1\n",
    "method = 'sum'\n",
    "\n",
    "# emsemble\n",
    "preds = ensemble(filenames, na_val, pad_len, thd, method)\n",
    "# write to file\n",
    "header = ['query-id', 'product1', 'product2', 'product3', 'product4', 'product5']\n",
    "with open(path+'prediction_{}_{}_{}_cls.csv'.format(method, len(filenames), thd), 'w', newline='') as f:\n",
    "    w = csv.writer(f)\n",
    "    w.writerow(header)\n",
    "    for qid in sorted(list(preds.keys())):\n",
    "        w.writerow([qid]+preds[qid])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mcan:visual = 55:14\n",
      "prediction_all_cls2_5119_38.csv: mean -> 0.195; std -> 0.254\n",
      "prediction_all_cls2_519_39.csv: mean -> 0.198; std -> 0.258\n",
      "prediction_all_cls2_123_36.csv: mean -> 0.2; std -> 0.261\n",
      "prediction_all_cls2_1234_36.csv: mean -> 0.212; std -> 0.24\n",
      "prediction_all_cls2_12345_31.csv: mean -> 0.197; std -> 0.257\n",
      "prediction_all_cls2_1213_34.csv: mean -> 0.198; std -> 0.258\n",
      "prediction_all_cls2_207_34.csv: mean -> 0.208; std -> 0.243\n",
      "prediction_all_cls2_1333_37.csv: mean -> 0.199; std -> 0.255\n",
      "prediction_all_cls2_2020_36.csv: mean -> 0.2; std -> 0.253\n",
      "prediction_all_cls2_1115_36.csv: mean -> 0.197; std -> 0.259\n",
      "prediction_all_cls2_666_32.csv: mean -> 0.199; std -> 0.254\n",
      "prediction_all_cls2_2574_39.csv: mean -> 0.198; std -> 0.254\n",
      "prediction_all_cls2_89983_38.csv: mean -> 0.202; std -> 0.258\n",
      "prediction_all_cls2_46555_38.csv: mean -> 0.198; std -> 0.258\n",
      "prediction_all_cls2_86031_39.csv: mean -> 0.196; std -> 0.257\n",
      "prediction_all_cls2_7414_35.csv: mean -> 0.207; std -> 0.235\n",
      "prediction_all_cls2_71438_38.csv: mean -> 0.199; std -> 0.258\n",
      "prediction_all_cls2_777_35.csv: mean -> 0.201; std -> 0.261\n",
      "prediction_all_cls2_87_35.csv: mean -> 0.199; std -> 0.257\n",
      "prediction_all_cls2_8787_32.csv: mean -> 0.205; std -> 0.26\n",
      "prediction_all_cls2_878787_30.csv: mean -> 0.199; std -> 0.259\n",
      "prediction_all_cls2_800061_31.csv: mean -> 0.205; std -> 0.246\n",
      "prediction_all_cls2_856710_31.csv: mean -> 0.195; std -> 0.255\n",
      "prediction_all_cls2_42_38.csv: mean -> 0.201; std -> 0.261\n",
      "prediction_all_cls2_426_36.csv: mean -> 0.202; std -> 0.254\n",
      "prediction_all_cls2_64_38.csv: mean -> 0.2; std -> 0.264\n",
      "prediction_all_cls2_8864_36.csv: mean -> 0.201; std -> 0.254\n",
      "prediction_all_cls2_26_39.csv: mean -> 0.203; std -> 0.255\n",
      "prediction_all_cls2_7382_39.csv: mean -> 0.202; std -> 0.252\n",
      "prediction_all_cls2_1010_39.csv: mean -> 0.201; std -> 0.256\n",
      "prediction_all_cls2_1001_36.csv: mean -> 0.196; std -> 0.259\n",
      "prediction_all_cls2_2330_37.csv: mean -> 0.204; std -> 0.246\n",
      "prediction_all_cls2_612_39.csv: mean -> 0.199; std -> 0.263\n",
      "prediction_all_cls2_24_38.csv: mean -> 0.199; std -> 0.254\n",
      "prediction_all_cls2_25_32.csv: mean -> 0.199; std -> 0.258\n",
      "prediction_all_cls2_2077_35.csv: mean -> 0.197; std -> 0.257\n",
      "prediction_all_cls2_2049_39.csv: mean -> 0.192; std -> 0.256\n",
      "prediction_all_cls2_2045_39.csv: mean -> 0.201; std -> 0.262\n",
      "prediction_all_cls2_1917_36.csv: mean -> 0.198; std -> 0.257\n",
      "prediction_all_cls2_78667_36.csv: mean -> 0.203; std -> 0.262\n",
      "prediction_all_cls2_68654_34.csv: mean -> 0.195; std -> 0.253\n",
      "prediction_all_cls2_56474_33.csv: mean -> 0.194; std -> 0.255\n",
      "prediction_all_cls2_56464_36.csv: mean -> 0.205; std -> 0.262\n",
      "prediction_all_cls2_54367_37.csv: mean -> 0.204; std -> 0.242\n",
      "prediction_all_cls2_4547_32.csv: mean -> 0.199; std -> 0.258\n",
      "prediction_all_cls2_437_36.csv: mean -> 0.2; std -> 0.258\n",
      "prediction_all_cls2_485_38.csv: mean -> 0.196; std -> 0.251\n",
      "prediction_all_cls2_132_38.csv: mean -> 0.194; std -> 0.253\n",
      "prediction_all_cls2_257_37.csv: mean -> 0.204; std -> 0.263\n",
      "prediction_all_cls2_584_35.csv: mean -> 0.197; std -> 0.254\n",
      "prediction_all_cls2_931_37.csv: mean -> 0.195; std -> 0.255\n",
      "prediction_all_cls2_792_33.csv: mean -> 0.2; std -> 0.244\n",
      "prediction_all_cls2_603_39.csv: mean -> 0.202; std -> 0.26\n",
      "prediction_all_cls2_746_39.csv: mean -> 0.2; std -> 0.253\n",
      "prediction_all_cls2_480_35.csv: mean -> 0.2; std -> 0.259\n",
      "prediction_all_cls2_413_32.csv: mean -> 0.203; std -> 0.264\n",
      "prediction_all_cls2_807_37.csv: mean -> 0.209; std -> 0.255\n",
      "prediction_all_cls2_9527_38.csv: mean -> 0.199; std -> 0.256\n",
      "prediction_all_cls2_713_36.csv: mean -> 0.207; std -> 0.252\n",
      "prediction_all_cls2_625_38.csv: mean -> 0.203; std -> 0.248\n",
      "prediction_all_cls2_1324_38.csv: mean -> 0.208; std -> 0.258\n",
      "prediction_all_cls2_987_34.csv: mean -> 0.206; std -> 0.25\n",
      "prediction_all_cls2_116_39.csv: mean -> 0.207; std -> 0.263\n",
      "prediction_all_cls2_41_30.csv: mean -> 0.201; std -> 0.259\n",
      "prediction_all_cls2_145_39.csv: mean -> 0.207; std -> 0.267\n",
      "prediction_all_cls2_7328_32.csv: mean -> 0.201; std -> 0.257\n",
      "prediction_all_cls2_62_35.csv: mean -> 0.203; std -> 0.256\n",
      "prediction_all_cls2_3951_37.csv: mean -> 0.201; std -> 0.259\n",
      "prediction_all_cls2_9736_38.csv: mean -> 0.197; std -> 0.257\n"
     ]
    }
   ],
   "source": [
    "means = []\n",
    "stds = []\n",
    "n_mcan = sum([1 for seed in list(mcans.keys()) if mcans[seed]])\n",
    "n_visual = sum([1 for seed in list(visuals.keys()) if visuals[seed]])\n",
    "print('mcan:visual = {}:{}'.format(n_mcan, n_visual))\n",
    "\n",
    "for filename in filenames:\n",
    "    qid2score = load_one_file(path+filename, na_val, pad_len, method)\n",
    "    scores = [score for qid in list(qid2score.keys()) \\\n",
    "              for score in list(qid2score[qid].values()) if score != na_val]\n",
    "    mean = sum(scores)/len(scores)\n",
    "    std = (sum([score**2 for score in scores])/len(scores) - mean**2)**0.5\n",
    "    means.append(mean)\n",
    "    stds.append(std)\n",
    "    print('{}: mean -> {}; std -> {}'.format(filename, round(mean, 3), round(std, 3)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 2.,  7., 10., 16., 14.,  7.,  5.,  6.,  1.,  1.]),\n",
       " array([0.19226365, 0.19425656, 0.19624948, 0.19824239, 0.2002353 ,\n",
       "        0.20222821, 0.20422113, 0.20621404, 0.20820695, 0.21019986,\n",
       "        0.21219278]),\n",
       " <a list of 10 Patch objects>)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAARH0lEQVR4nO3de7CcdX3H8fdXQtQoCiFHpODxQAfooOOtR+tlqnKxRrDGTp0pTLGgOKdqtba1tbFOy9Sp03iZio5tnQzG4EjjBam1RKwpwmCnXEwoYLgoiCkG0QSpN2ylqd/+8fwObJaTnN19nrPn+Mv7NbNzdn/P5ffN75x89rfP8+xuZCaSpLo8YrELkCR1z3CXpAoZ7pJUIcNdkipkuEtShZaNs7NVq1bl1NTUOLuUpJ9727ZtuzczJ4bZZqzhPjU1xdatW8fZpST93IuI/xx2Gw/LSFKFDHdJqpDhLkkVMtwlqUKGuyRVyHCXpArNG+4RsSEidkXE9r72N0fEbRFxc0S8Z+FKlCQNa5CZ+0ZgdW9DRJwErAGenplPAd7XfWmSpFHNG+6ZeRVwX1/zG4B1mfnTss6uBahNkjSiUd+hejzwqxHxLuB/gD/OzK/MtWJEzAAzAJOTkyN2p3GaWrt50frese70RetbqsmoJ1SXASuB5wJ/AnwqImKuFTNzfWZOZ+b0xMRQH40gSRrRqOG+E7gkG9cBPwNWdVeWJKmNUcP9s8BJABFxPLAcuLeroiRJ7cx7zD0iNgEvBlZFxE7gPGADsKFcHvkAcHb6TduStGTMG+6ZeeY+Fp3VcS2SpI74DlVJqpDhLkkVMtwlqUKGuyRVyHCXpAoZ7pJUIcNdkipkuEtShQx3SaqQ4S5JFTLcJalChrskVchwl6QKGe6SVCHDXZIqZLhLUoXmDfeI2BARu8q3LvUve2tEZET4/amStIQMMnPfCKzub4yIJwG/BtzVcU2SpJbmDffMvAq4b45F7wfeBvjdqZK0xIx0zD0i1gB3Z+aNHdcjSerAvF+Q3S8iVgB/RnNIZpD1Z4AZgMnJyWG70wFmau3mRel3x7rTF6VfaaGMMnP/ReAY4MaI2AEcDVwfEU+ca+XMXJ+Z05k5PTExMXqlkqSBDT1zz8yvAk+YfVwCfjoz7+2wLklSC4NcCrkJuBo4ISJ2RsS5C1+WJKmNeWfumXnmPMunOqtGktQJ36EqSRUy3CWpQoa7JFXIcJekChnuklQhw12SKmS4S1KFDHdJqpDhLkkVMtwlqUKGuyRVyHCXpAoZ7pJUIcNdkipkuEtShQx3SaqQ4S5JFRrka/Y2RMSuiNje0/beiLgtIm6KiH+MiEMXtkxJ0jAGmblvBFb3tW0BnpqZTwO+Dry947okSS3MG+6ZeRVwX1/bFzNzT3l4DXD0AtQmSRpRF8fcXwtctq+FETETEVsjYuvu3bs76E6SNJ9W4R4R7wD2ABfta53MXJ+Z05k5PTEx0aY7SdKAlo26YUScA7wcOCUzs7OKJEmtjRTuEbEaeBvwosz8SbclSZLaGuRSyE3A1cAJEbEzIs4FPgQcAmyJiBsi4sMLXKckaQjzztwz88w5mj+yALVIkjriO1QlqUKGuyRVyHCXpAoZ7pJUIcNdkipkuEtShQx3SaqQ4S5JFTLcJalChrskVchwl6QKGe6SVCHDXZIqZLhLUoUMd0mqkOEuSRUy3CWpQoN8zd6GiNgVEdt72lZGxJaIuL38PGxhy5QkDWOQmftGYHVf21rg8sw8Dri8PJYkLRHzhntmXgXc19e8Briw3L8QeGXHdUmSWpj3C7L34YjMvKfc/w5wxL5WjIgZYAZgcnJyxO4OTFNrNy92CZJ+TrU+oZqZCeR+lq/PzOnMnJ6YmGjbnSRpAKOG+3cj4kiA8nNXdyVJktoaNdw/B5xd7p8N/FM35UiSujDIpZCbgKuBEyJiZ0ScC6wDXhIRtwOnlseSpCVi3hOqmXnmPhad0nEtkqSO+A5VSaqQ4S5JFTLcJalChrskVchwl6QKGe6SVCHDXZIqZLhLUoUMd0mqkOEuSRUy3CWpQoa7JFXIcJekChnuklQhw12SKmS4S1KFDHdJqlCrcI+IP4yImyNie0RsiohHdVWYJGl0I4d7RBwF/D4wnZlPBQ4CzuiqMEnS6NoellkGPDoilgErgG+3L0mS1NbI4Z6ZdwPvA+4C7gF+kJlf7F8vImYiYmtEbN29e/folUqSBtbmsMxhwBrgGOAXgMdExFn962Xm+syczszpiYmJ0SuVJA2szWGZU4FvZubuzPxf4BLg+d2UJUlqo0243wU8NyJWREQApwC3dlOWJKmNNsfcrwUuBq4Hvlr2tb6juiRJLSxrs3Fmngec11EtkqSO+A5VSaqQ4S5JFTLcJalChrskVchwl6QKGe6SVCHDXZIq1Oo69wPF1NrNi12CFthi/o53rDt90fpWvZy5S1KFDHdJqpDhLkkVMtwlqUKGuyRVyHCXpAoZ7pJUIcNdkipkuEtShVqFe0QcGhEXR8RtEXFrRDyvq8IkSaNr+/EDHwC+kJmviojlwIoOapIktTRyuEfE44EXAucAZOYDwAPdlCVJaqPNzP0YYDfw0Yh4OrANeEtm3t+7UkTMADMAk5OTLbqT1CU/LK1ubY65LwOeBfx9Zj4TuB9Y279SZq7PzOnMnJ6YmGjRnSRpUG3CfSewMzOvLY8vpgl7SdIiGzncM/M7wLci4oTSdApwSydVSZJaaXu1zJuBi8qVMncCr2lfkiSprVbhnpk3ANMd1SJJ6ojvUJWkChnuklQhw12SKmS4S1KFDHdJqpDhLkkVMtwlqUJt38QkqaXF/AAv1cuZuyRVyHCXpAoZ7pJUIcNdkipkuEtShQx3SaqQ4S5JFTLcJalChrskVah1uEfEQRHxHxFxaRcFSZLa62Lm/hbg1g72I0nqSKtwj4ijgdOBC7opR5LUhbYz9/OBtwE/29cKETETEVsjYuvu3btbdidJGsTI4R4RLwd2Zea2/a2XmeszczozpycmJkbtTpI0hDYz9xcAr4iIHcAngJMj4uOdVCVJamXkcM/Mt2fm0Zk5BZwBfCkzz+qsMknSyLzOXZIq1Mk3MWXmlcCVXexLktSeM3dJqpDhLkkVMtwlqUKGuyRVyHCXpAoZ7pJUIcNdkipkuEtShQx3SaqQ4S5JFTLcJalChrskVchwl6QKGe6SVCHDXZIqZLhLUoUMd0mq0MjhHhFPiogrIuKWiLg5It7SZWGSpNG1+Zq9PcBbM/P6iDgE2BYRWzLzlo5qkySNaOSZe2bek5nXl/s/Am4FjuqqMEnS6Dr5guyImAKeCVw7x7IZYAZgcnJy5D6m1m4eeVtJOtC0PqEaEY8FPgP8QWb+sH95Zq7PzOnMnJ6YmGjbnSRpAK3CPSIOpgn2izLzkm5KkiS11eZqmQA+AtyamX/TXUmSpLbazNxfALwaODkibii30zqqS5LUwsgnVDPz34DosBZJUkd8h6okVchwl6QKGe6SVCHDXZIqZLhLUoUMd0mqkOEuSRXq5IPDJGkYB+IHAe5Yd/pY+3PmLkkVMtwlqUKGuyRVyHCXpAoZ7pJUIcNdkipkuEtShQx3SaqQ4S5JFWr7BdmrI+JrEXFHRKztqihJUjttviD7IOBvgZcBJwJnRsSJXRUmSRpdm5n7c4A7MvPOzHwA+ASwppuyJElttPngsKOAb/U83gn8Sv9KETEDzJSHP46Ir7Xos9cq4N6O9tUl6xrOUq0Llm5t1jWcJVFXvPthTcPU9eRh+1vwT4XMzPXA+q73GxFbM3O66/22ZV3DWap1wdKtzbqGc6DW1eawzN3Ak3oeH13aJEmLrE24fwU4LiKOiYjlwBnA57opS5LUxsiHZTJzT0S8CfgX4CBgQ2be3Fll8+v8UE9HrGs4S7UuWLq1WddwDsi6IjMXcv+SpEXgO1QlqUKGuyTVKDMX5QasBr4G3AGsnWP5C4HrgT3Aq/qWvRvYXm6/1dN+UdnndmADcHBpfzHwA+CGcvuLMde1EfhmT//PKO0BfLD0dRPwrDHX9eWemr4NfHbY8Rqwtj8Cbin/xsuBJ/csOxu4vdzO7mn/ZeCrZZ8f5KFDiCuBLWX9LcBh46oLWAFsBm4DbgbW9ax/DrC7Z8xeN+bxurLsc7b/J5T2RwKfLH1dC0yNcbwO6annBpprus8f83h9Afg+cGnfNseU8bijjM/yMY/XvupqnWEP7mu+FRbiRnMC9hvAscBy4EbgxL51poCnAR+jJ6yA02n+Uy8DHkNz1c7jyrLTaAIzgE3AG3oG5tJFrGsjfYHbU+9lpd7nAteOs66+7T8D/M4w4zVEbScBK8r9NwCfLPdXAneWn4eV+4eVZdeVMYkyRi8r7e+h/EcC1gLvHlddNOF+UllnOc2T42xd5wAfWsTxuhKYnqO/NwIfLvfPmN3XuOrq234b8MJxjVd5fArw6zw8RD8FnFHuf5iHsmLBx2ueulplWO9tsQ7LzPvRBZm5IzNvAn7Wt+2JwFWZuScz76d5Vlxdtvl8FjThcPRSqGs/1gAfKyVfAxwaEUeOu66IeBxwMvDZeeqdyyC1XZGZPykPr+Gh38tLgS2ZeV9m/hfNk9DqMgaPy8xryu/yY8AryzZrgAvL/Qt72he8rsz8SWZeUbZ9gOaV0kL8jQ1V1zz99Y7XxcApERHjrisijgeeQPOEOIw2dZGZlwM/6qslaP7eLy5NvX9H4xivOesq7W0z7EGLFe5zfXTBUQNueyNNAKyIiFU0z469b6YiIg4GXk3z0mfW8yLixoi4LCKesgh1vSsiboqI90fEI4fsb0HHi+YP+/LM/GFP2yDjNUpt59LMxPe37VHl/lz7PCIz7yn3vwMcMca6HhQRh9LMvC7vaf7N8ju+OCL6x3gcdX00Im6IiD/vCaQHt8nMPTQv7Q8fc13w0Cy49/K8hR6vfTkc+H4Zj/59jmO85tUiwx604B8/0LXM/GJEPBv4d5pjdlcD/9e32t/RzFZnZwnX0xzv+nFEnEYzQz1ujHW9nSaIltNc2/qnwDu77H/EumadCVzQ83hBxisizgKmgRe13RdAZmZEtL6Wd9i6ImIZzUvmD2bmnaX5n4FNmfnTiPhdmtnfyWOs67cz8+6IOITmENuraV7xdG7E3+MZpaZZiz1eYzNiXa0zbLFm7q0+uiAz35WZz8jMl9Acm/r67LKIOA+YoDmZMbv+DzPzx+X+54GDyyx2LHVl5j3lldZPgY/SvKQbpr+FHK9VpZ7NPesPOl4D1xYRpwLvAF5RxmF/297N3i9He/f53dlDV+XnrjHWNWs9cHtmnj/bkJnf69n+ApoTwmOrKzNnf/4I+Afm+BsrT0qPB743rrrKNk8HlmXmttm2MY3XvnyP5hDo7OS2d5/jGK/9aplhD8khDtB3daN5xXAnzRnr2ZMRT9nHuhvZ+wThQcDh5f7TaM4qLyuPX0czQ3103z6eyENXWzwHuGv28ZjqOrL8DOB8ylUWNCc7e0+oXjfO8SptrwcuHGW8Bq0NeCbNyafj+tpX0lxFdFi5fRNYWZb1n1A9rbS/l71PqL5nzHX9Fc3M+BF92xzZc/83gGvGVVfZ56qyzsE0x4pfXx7/HnufIPzUOMerLF8H/OW4x6tn+Yt5+InLT7P3CdU3jmu85qmrVYbttc3+Fi7kjeas8NfLP/4dpe2dNM9wAM+mOY51P80z582l/VE0lxfdQnOS4hk9+9xT9rfX5ULAm2guXbuxbPP8Mdf1JZrL+rYDHwceW9qD5gtPvlGWP+xqh4Wsqyy/kuZkYW/bwOM1YG3/Cny35/fyuZ5tX0tzKdkdwGt62qfLeH0D+FDPH/bhNMe5by/7XTmuumhmZwncSt8lfMBf94zZFcAvjbGux9BciXJTqeEDwEE9v/9Pl/WvA44d5++xLLuzfzzGOF5fpjkc+d80/z9eWtqPLeNxRxmfR455vPZVV+sMm7358QOSVCHfoSpJFTLcJalChrskVchwl6QKGe6SVCHDXZIqZLhLUoX+HzMXgG4bwAhDAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(means)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 1.,  1.,  3.,  2.,  2.,  9., 19., 20.,  9.,  3.]),\n",
       " array([0.23514857, 0.23829719, 0.24144581, 0.24459443, 0.24774306,\n",
       "        0.25089168, 0.2540403 , 0.25718893, 0.26033755, 0.26348617,\n",
       "        0.26663479]),\n",
       " <a list of 10 Patch objects>)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAASZ0lEQVR4nO3dfYxld33f8fcnfmprk2Kzk43x00LiujJJbOh0gYZEPNmxF4RphVq7KbFbRxuIkYKaqHWKGldElRxRQh42irXFi01LHPKAyUo24K0DcpAwMHbX9hpDvLgbeRfHO2CwcYmCln77xz0L18Od3Tv33Lkz88v7JV3dc36/3znne87Mfubsufeem6pCktSuH1jrAiRJq8ugl6TGGfSS1DiDXpIaZ9BLUuNOXOsCRtm0aVNt2bJlrcuQpA3jvvvu+2pVzY3qW5dBv2XLFhYWFta6DEnaMJL81XJ9XrqRpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTtu0Cc5J8knk3whycNJfqlrPyPJniSPds+nL7P81d2YR5NcPe0dkCQd2zhn9EeAX66qC4FXANcluRC4Hri7qs4H7u7mnyPJGcANwMuBrcANy/1BkCStjuMGfVU9UVX3d9PfBB4BzgKuAG7tht0KvHnE4j8D7Kmqp6rq68Ae4LJpFC5JGs+KPhmbZAvwUuCzwOaqeqLr+mtg84hFzgIeH5o/2LWNWvd2YDvAueeeu5KyJK2iLdffsSbbPXDjG9Zkuy0a+8XYJKcBfwq8s6qeGe6rwddU9fqqqqraWVXzVTU/Nzfydg2SpAmMFfRJTmIQ8h+qqo90zU8mObPrPxM4PGLRQ8A5Q/Nnd22SpBkZ5103AW4GHqmq3xzq2g0cfRfN1cCfjVj8E8ClSU7vXoS9tGuTJM3IOGf0Pwm8FXhtkr3dYxtwI3BJkkeB13fzJJlP8n6AqnoK+HXg893j3V2bJGlGjvtibFV9Gsgy3a8bMX4B+Pmh+V3ArkkLlCT14ydjJalxBr0kNc6gl6TGGfSS1Lh1+Z2xkp5rrT6dqjZ4Ri9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXuuPe6SbILeCNwuKp+rGv7MHBBN+T5wDeq6uIRyx4Avgl8BzhSVfNTqluSNKZxbmp2C7AD+ODRhqr6V0enk7wXePoYy7+mqr46aYGSpH7G+SrBe5JsGdXXfXH4vwReO92yJEnT0vca/U8BT1bVo8v0F3BXkvuSbO+5LUnSBPrej/4q4LZj9L+qqg4l+SFgT5IvVtU9owZ2fwi2A5x77rk9y5IkHTXxGX2SE4F/AXx4uTFVdah7PgzcDmw9xtidVTVfVfNzc3OTliVJWqLPpZvXA1+sqoOjOpOcmuR5R6eBS4F9PbYnSZrAcYM+yW3AZ4ALkhxMcm3XdSVLLtskeWGSO7vZzcCnkzwAfA64o6o+Pr3SJUnjGOddN1ct037NiLavANu66ceAi3rWJ0nqyU/GSlLjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuPG+SrBXUkOJ9k31PZfkhxKsrd7bFtm2cuSfCnJ/iTXT7NwSdJ4xjmjvwW4bET7+6rq4u5x59LOJCcAvwdcDlwIXJXkwj7FSpJW7rhBX1X3AE9NsO6twP6qeqyqvg38IXDFBOuRJPXQ5xr9O5I82F3aOX1E/1nA40PzB7u2kZJsT7KQZGFxcbFHWZKkYZMG/e8DPwJcDDwBvLdvIVW1s6rmq2p+bm6u7+okSZ2Jgr6qnqyq71TV/wP+O4PLNEsdAs4Zmj+7a5MkzdBEQZ/kzKHZfw7sGzHs88D5SV6U5GTgSmD3JNuTJE3uxOMNSHIb8GpgU5KDwA3Aq5NcDBRwAPiFbuwLgfdX1baqOpLkHcAngBOAXVX18KrshSRpWccN+qq6akTzzcuM/QqwbWj+TuD73nopSZodPxkrSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTtu0CfZleRwkn1Dbe9J8sUkDya5Pcnzl1n2QJKHkuxNsjDNwiVJ4xnnjP4W4LIlbXuAH6uqnwD+EvjVYyz/mqq6uKrmJytRktTHcYO+qu4BnlrSdldVHelm7wXOXoXaJElTMI1r9P8O+NgyfQXcleS+JNuPtZIk25MsJFlYXFycQlmSJOgZ9EneBRwBPrTMkFdV1cuAy4Hrkvz0cuuqqp1VNV9V83Nzc33KkiQNmTjok1wDvBH42aqqUWOq6lD3fBi4Hdg66fYkSZOZKOiTXAb8B+BNVfWtZcacmuR5R6eBS4F9o8ZKklbPOG+vvA34DHBBkoNJrgV2AM8D9nRvnbypG/vCJHd2i24GPp3kAeBzwB1V9fFV2QtJ0rJOPN6AqrpqRPPNy4z9CrCtm34MuKhXdZKk3vxkrCQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDVurKBPsivJ4ST7htrOSLInyaPd8+nLLHt1N+bRJFdPq3BJ0njGPaO/BbhsSdv1wN1VdT5wdzf/HEnOAG4AXg5sBW5Y7g+CJGl1jBX0VXUP8NSS5iuAW7vpW4E3j1j0Z4A9VfVUVX0d2MP3/8GQJK2iPtfoN1fVE930XwObR4w5C3h8aP5g1/Z9kmxPspBkYXFxsUdZkqRhU3kxtqoKqJ7r2FlV81U1Pzc3N42yJEn0C/onk5wJ0D0fHjHmEHDO0PzZXZskaUb6BP1u4Oi7aK4G/mzEmE8AlyY5vXsR9tKuTZI0I+O+vfI24DPABUkOJrkWuBG4JMmjwOu7eZLMJ3k/QFU9Bfw68Pnu8e6uTZI0IyeOM6iqrlqm63Ujxi4APz80vwvYNVF1kqTe/GSsJDXOoJekxhn0ktQ4g16SGmfQS1LjxnrXjaSBLdffsdYl/J2xlsf6wI1vWLNtrwbP6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMZNHPRJLkiyd+jxTJJ3Lhnz6iRPD435tf4lS5JWYuKbmlXVl4CLAZKcABwCbh8x9C+q6o2TbkeS1M+0Lt28DvhyVf3VlNYnSZqSaQX9lcBty/S9MskDST6W5CXLrSDJ9iQLSRYWFxenVJYkqXfQJzkZeBPwxyO67wfOq6qLgN8FPrrceqpqZ1XNV9X83Nxc37IkSZ1pnNFfDtxfVU8u7aiqZ6rq2W76TuCkJJumsE1J0pimEfRXscxlmyQ/nCTd9NZue1+bwjYlSWPq9VWCSU4FLgF+YajtbQBVdRPwFuDtSY4AfwNcWVXVZ5uSpJXpFfRV9X+BFyxpu2loegewo882JEn9+MlYSWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJalzvoE9yIMlDSfYmWRjRnyS/k2R/kgeTvKzvNiVJ4+v1VYJDXlNVX12m73Lg/O7xcuD3u2dJ0gzM4tLNFcAHa+Be4PlJzpzBdiVJTCfoC7gryX1Jto/oPwt4fGj+YNf2HEm2J1lIsrC4uDiFsiRJMJ2gf1VVvYzBJZrrkvz0JCupqp1VNV9V83Nzc1MoS5IEUwj6qjrUPR8Gbge2LhlyCDhnaP7srk2SNAO9gj7JqUmed3QauBTYt2TYbuDnunffvAJ4uqqe6LNdSdL4+r7rZjNwe5Kj6/qDqvp4krcBVNVNwJ3ANmA/8C3g3/bcpiRpBXoFfVU9Blw0ov2moekCruuzHUnS5PxkrCQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY3r+8UjWge2XH/Hmmz3wI1vWJPtSquttX9TntFLUuMmDvok5yT5ZJIvJHk4yS+NGPPqJE8n2ds9fq1fuZKklepz6eYI8MtVdX/3BeH3JdlTVV9YMu4vquqNPbYjSeph4jP6qnqiqu7vpr8JPAKcNa3CJEnTMZVr9Em2AC8FPjui+5VJHkjysSQvOcY6tidZSLKwuLg4jbIkSUwh6JOcBvwp8M6qemZJ9/3AeVV1EfC7wEeXW09V7ayq+aqan5ub61uWJKnTK+iTnMQg5D9UVR9Z2l9Vz1TVs930ncBJSTb12aYkaWX6vOsmwM3AI1X1m8uM+eFuHEm2dtv72qTblCStXJ933fwk8FbgoSR7u7b/BJwLUFU3AW8B3p7kCPA3wJVVVT22KUlaoYmDvqo+DeQ4Y3YAOybdhta3tfr0oKSV8ZOxktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUuOa+M9ZPa0rSc3lGL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4/p+OfhlSb6UZH+S60f0n5Lkw13/Z5Ns6bM9SdLK9fly8BOA3wMuBy4Erkpy4ZJh1wJfr6ofBd4H/Mak25MkTabPGf1WYH9VPVZV3wb+ELhiyZgrgFu76T8BXpfkmN8zK0marj63QDgLeHxo/iDw8uXGVNWRJE8DLwC+unRlSbYD27vZZ5N8aYX1bBq13g1mo++D9a+9jb4Pf6frT79rHuct17Fu7nVTVTuBnZMun2ShquanWNLMbfR9sP61t9H3wfpXR59LN4eAc4bmz+7aRo5JciLwD4Gv9dimJGmF+gT954Hzk7woycnAlcDuJWN2A1d3028B/ryqqsc2JUkrNPGlm+6a+zuATwAnALuq6uEk7wYWqmo3cDPwP5LsB55i8MdgtUx82Wcd2ej7YP1rb6Pvg/WvgniCLUlt85OxktQ4g16SGrdug36M2yv8+yRfSPJgkruTnNe1n5fk/iR7kzyc5G1Dy3yqW+fe7vFD663+of4fTHIwyY6htn+S5KFunb+zmh8+W6X6Z3b8++5Dku8M1bl7qP1F3e089ne39zh5g9V/S5L/M9R38Tqt/9wkdyV5pBuzpWuf2fFfxX2Y2c/gu6pq3T0YvLj7ZeDFwMnAA8CFS8a8BvgH3fTbgQ930ycDp3TTpwEHgBd2858C5tdz/UP9vw38AbBjqO1zwCuAAB8DLt9g9c/k+E9jH4Bnl1nvHwFXdtM3AW/fYPXfArxlAxz/TwGXdNOnDY2byfFf5X2Yyc9g+LFez+iPe3uFqvpkVX2rm72Xwfv4qapvV9Xfdu2nsDb/a5m4fhicuQObgbuG2s4EfrCq7q3Bb8sHgTdvlPrXQK99GKX7H9RrGdzOAwa391iXP4N1YOL6M7hn1olVtacb92xVfWvGx39V9mEVaz2m9Rr0o26vcNYxxl/L4AwXgCTnJHmwW8dvVNVXhsZ+oPvv0n9exUsfE9ef5AeA9wK/MmKdB1ewzj5Wo/6jZnH8oefvEPD3kiwkuTfJ0TB5AfCNqjoy5jr7WI36j/qv3aWG9yU5ZUr1LtWn/n8EfCPJR5L87yTvyeAmirM8/rA6+3DULH4G37Veg35sSf4NMA+852hbVT1eVT8B/ChwdZLNXdfPVtWPAz/VPd4663qXGlH/LwJ3VtXB5ZdaP1ZY/7o7/jD6dwg4rwYfZf/XwG8l+ZE1KW4MK6z/V4F/DPxT4AzgP86y1lFG1H8ig9+PX2FQ54uBa9akuDGtcB9m/jNYr0E/zu0VSPJ64F3Am4Yu13xXdya/j8EBp6oOdc/fZHD9eOvUKx/oU/8rgXckOQD8N+DnktzYLT/8X/OR65yS1ah/lscfev4ODdX6GINrrS9lcPuO52dwO49l1zklq1E/VfVEDfwt8AHW57+Bg8De7pLJEeCjwMuY7fGH1dmHWf4MvmeWLwiM+2Dw1/Ax4EV870WQlywZ81IGL5Scv6T9bODvd9OnA38J/Hi3zk1d+0kMrvO9bb3Vv2TMNRz7xdhtG6X+WR7/KfwOnc73XtDfBDxK9yIc8Mc898XAX9xg9Z/ZPQf4LeDGdVj/Cd34uW7+A8B1szz+q7wPM/kZPKee1d5Aj4O8jUFIfxl4V9f2bgZ/NQH+F/AksLd77O7aLwEe7A7yg8D2rv1U4L6u7WEG7wo5Yb3Vv2Qd1/DcoJ9n8D+ULwM76D7ZvBHqn/Xx7/k79M+Ah7rfoYeAa4fW+WIGf3D3MwidUzZY/X/ete0D/idw2nqrv+s7+u/4IQbvUjl51sd/FfdhZj+Dow9vgSBJjVuv1+glSVNi0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TG/X8ftEX6+VIxvAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(stds)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
